import time
from urllib.parse import urlparse,parse_qsl
import pandas as pd

#时间处理
def datetime_to_timestamp(dt):
    #时间 -> 时间戳
    return int(time.mktime(time.strptime(dt, "%Y-%m-%d %H:%M:%S")))

def df_time_to_timestamp(row):
    return datetime_to_timestamp(row['time'])

exclude_topdomains = ['com.cn','org.cn','net.cn']

def parse_url(url):
    p = urlparse(url)
    q = parse_qsl(p.query)
    return { 'scheme':p.scheme,'hostname':p.netloc,'query':dict(q) }

def find_query_field(q,fields):
    for field in fields:
        if field in q:
            return q[field]
    return ''

def find_query_kwd(q):
    return ''

def df_fromurl_to_hostname(row):
    p = parse_url(row['fromurl'])
    return p['hostname']

def df_hostname_to_domain(row):
    host = row['hostname']
    lst = host.split('.')
    l = len(lst)
    if l<3 :
        return host
    else:
        domain = lst[l-2]+'.'+lst[l-1]
        if domain in exclude_topdomains:
            return lst[l-3]+'.'+domain
        else:
            return domain

def df_fromurl_to_sid(row):
    p = parse_url(row['fromurl'])
    return find_query_field(p['query'],['sid','Sid','sourceid'])

def df_fromurl_to_JJID(row):
    p = parse_url(row['fromurl'])
    return find_query_field(p['query'],['JJID'])

def df_fromurl_to_mediaid(row):
    p = parse_url(row['fromurl'])
    return find_query_field(p['query'],['mediaid'])

def df_fromurl_to_kwd(row):
    p = parse_url(row['fromurl'])
    return find_query_kwd(p['query'])



ly_data_df = pd.read_csv('data_merge/csv/ly_data.csv')
ly_data_df.insert(loc=0,column='timestamp',value=ly_data_df.apply(df_time_to_timestamp,axis='columns'))
ly_data_df.insert(loc=0,column='hostname',value=ly_data_df.apply(df_fromurl_to_hostname,axis='columns'))
ly_data_df.insert(loc=0,column='domain',value=ly_data_df.apply(df_hostname_to_domain,axis='columns'))
ly_data_df.insert(loc=0,column='sid',value=ly_data_df.apply(df_fromurl_to_sid,axis='columns'))
ly_data_df.insert(loc=0,column='JJID',value=ly_data_df.apply(df_fromurl_to_JJID,axis='columns'))
ly_data_df.insert(loc=0,column='mediaid',value=ly_data_df.apply(df_fromurl_to_mediaid,axis='columns'))
ly_data_df.insert(loc=0,column='kwd',value=ly_data_df.apply(df_fromurl_to_kwd,axis='columns'))

ly_data_df = ly_data_df.drop(['type','custom_id','siteinfo_id','id','time','ip','industry_id2','industry_id'],1)

# print(ly_data_df.columns)

ly_data_list = ly_data_df.to_dict(orient='records')

ly = ly_data_list[0]
print(ly)

# for ly in ly_data_list:
    # print(ly['hostname'],' => ',ly['domain'])